from __future__ import unicode_literals
import requests
from bs4 import BeautifulSoup
import time
import json

urllist=[]
mp4u=[]
base_url='https://www.pornhdprime.com'
start_url='https://www.pornhdprime.com/category'
r=requests.get(start_url,timeout=5)
content=r.content
soup=BeautifulSoup(content,'lxml')
for item in soup.find('ul',attrs={'class':'tag-150-list'}).find_all('a'):
    urllist.append(base_url+item['href'])
    
while True:
    time.sleep(2)
    if len(urllist)==0:
        break
    nexturl=urllist.pop()
    print(nexturl+'\t'+str(len(urllist)))
    try:
        rr=requests.get(nexturl,timeout=5)
    except Exception as e:
        continue
    ccontent=rr.content
    ssoup=BeautifulSoup(ccontent,'lxml')
    for item in ssoup.find('ul',attrs={'class':'thumbs'}).find_all('li'):
        try:
            mp4url=item['data-mp4']
            if mp4url in mp4u:
                continue
            mp4u.append(mp4url)  
            with open('prime.txt','a',encoding='utf-8') as f:
                f.write(mp4url+'\n')
        except Exception as e:
            continue
    try:
        newurl=ssoup.find('li',attrs={'class':'next'}).find('span')['data-query-value']
        if int(newurl)>2 and int(newurl)<=10:
            newurl=nexturl[:len(nexturl)-1]+newurl
        elif int(newurl)>10:
            newurl=nexturl[:len(nexturl)-len(newurl)]+newurl
        else:
            newurl=nexturl+'?page='+newurl
        urllist.append(newurl)
        print('newurl:\t'+newurl)
    except Exception as e:
        print(e)

        continue
